{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import array\n",
    "from pickle import dump\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    " \n",
    "# load doc into memory\n",
    "def load_doc(filename):\n",
    "    file = open(filename, 'r')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text\n",
    " \n",
    "# load\n",
    "in_filename = 'data/clean_sequences.txt'\n",
    "doc = load_doc(in_filename)\n",
    "lines = doc.split('\\n')\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(lines)\n",
    "sequences = tokenizer.texts_to_sequences(lines)\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    " \n",
    "# separate into input and output\n",
    "sequences = array(sequences)\n",
    "X, y = sequences[:,:-1], sequences[:,-1]\n",
    "y = to_categorical(y, num_classes=vocab_size)\n",
    "seq_length = X.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(Embedding(vocab_size, 50, input_length=seq_length))\n",
    "model.add(LSTM(100, return_sequences=True))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dense(vocab_size, activation='softmax'))\n",
    "print(model.summary())\n",
    "# compile model\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# fit model\n",
    "model.fit(X, y, batch_size=128, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model to file\n",
    "model.save('model.h5')\n",
    "# save the tokenizer\n",
    "dump(tokenizer, open('tokenizer.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def texts_to_sequences(texts, word_to_index):\n",
    "    indices = list()\n",
    "    \n",
    "    for text in texts:\n",
    "        indices.append(word_to_index[text])\n",
    "        \n",
    "    return indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "(1, 50)\n",
      "assumptions courtney jokers rush crawl apartment gutter pharaoh extension bushes pharaoh apartment jangalang knock funny ball cannot sleepy kitchen marino think fewest grey funerals apartment curse knock fucker floors worn jawn apartment demanded jokers apartment chaining best wasted funerals cannot unpunished fuckin apartment narcotics breathin clown apartment possessed crawl apartment\n"
     ]
    }
   ],
   "source": [
    "from random import randint\n",
    "from pickle import load\n",
    "from keras.models import load_model\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def load_doc(filename):\n",
    "    file = open(filename, 'r')\n",
    "    text = file.read()\n",
    "    file.close()\n",
    "    return text\n",
    "\n",
    "def generate_seq(model, tokenizer, seq_length, seed_text, n_words):\n",
    "    result = list()\n",
    "    in_text = seed_text\n",
    "    \n",
    "    for _ in range(n_words):\n",
    "        encoded = tokenizer.texts_to_sequences([in_text])[0]\n",
    "#         print(encoded)\n",
    "        encoded = pad_sequences([encoded], maxlen=seq_length, truncating='pre')\n",
    "#         print(encoded)\n",
    "        print(encoded.shape)\n",
    "        \n",
    "        yhat = model.predict_classes(encoded, verbose=0)\n",
    "        out_word = ''\n",
    "    \n",
    "        for word, index in tokenizer.word_index.items():\n",
    "            if index == yhat:\n",
    "                out_word = word\n",
    "                break\n",
    "        \n",
    "        in_text += ' ' + out_word\n",
    "        result.append(out_word)\n",
    "    \n",
    "    return ' '.join(result)\n",
    "\n",
    "doc = load_doc(in_filename)\n",
    "lines = doc.split('\\n')\n",
    "seq_length = len(lines[0].split()) - 1\n",
    "\n",
    "model = load_model('model.h5')\n",
    "\n",
    "tokenizer = load(open('tokenizer.pkl', 'rb'))\n",
    "\n",
    "seed_text = lines[randint(0,len(lines))]\n",
    "# print(seed_text + '\\n')\n",
    "\n",
    "generated = generate_seq(model, tokenizer, seq_length, seed_text, 50)\n",
    "print(generated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
